package com.example.demo.com.task;

import com.example.demo.com.crawler.pojo.News;
import com.example.demo.com.task.SpringDataPipeline;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Component;
import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.processor.PageProcessor;
import us.codecraft.webmagic.scheduler.BloomFilterDuplicateRemover;
import us.codecraft.webmagic.scheduler.QueueScheduler;

/**
 * <h3>webmagic-scheduled</h3>
 * <p></p>
 *
 * @author : 你的名字
 * @date : 2020-05-14 07:40
 **/
@Component
public class NewsProcesser implements PageProcessor {
    private static final String URL = "https://www.jhc.cn/";

    @Autowired
    private SpringDataPipeline dataPipeline;


    @Override
    public void process(Page page) {
        if (!page.getUrl().toString().contains("page.htm")) {
            page.addTargetRequests(page.getHtml().xpath("//li[@class='news-item']").links().all());
        } else {
            News newInfo = new News();
            String url = page.getUrl().toString();
            newInfo.setUrl(url);

            String title = page.getHtml().xpath("//h1[@class='arti-title']/text()").toString();
            newInfo.setTitle(title);

            String[] source = page.getHtml().xpath("//p[@class='arti-metas']/span[1]/text()").toString().split("：");
            newInfo.setSource(source[1]);

            String[] auther = page.getHtml().xpath("//p[@class='arti-metas']/span[2]/text()").toString().split("：");
            newInfo.setAuthor(auther[1]);

            String[] date = page.getHtml().xpath("//p[@class='arti-metas']/span[3]/text()").toString().split("：");
            newInfo.setReleaseDate(date[1]);

            String view = page.getHtml().xpath("//p[@class='arti-metas']/span[4]/span/text()").toString();
            newInfo.setPageViews(view);

            String content = page.getHtml().xpath("//div[@class='wp_articlecontent']").toString();
//                //剔除带class属性div
            String replace = content.replace("<div class=\"wp_articlecontent\">", "");
            String replace2 = replace.replace("</div>", "");
            newInfo.setContent(replace2);

            page.putField("infos", newInfo);
        }
    }

    private Site site = Site.me()
            .setCharset("utf8")
            //超时时间
            .setTimeOut(10 * 1000)
            //重试间隔时间
            .setRetrySleepTime(3000)
            //重试次数
            .setRetryTimes(3);

    @Override
    public Site getSite() {
        return site;
    }

    @Scheduled(initialDelay = 1000, fixedDelay = 100 * 1000)
    public void runCrawler() {
        Spider.create(new NewsProcesser())
                .addUrl(URL)
                .setScheduler(new QueueScheduler().setDuplicateRemover(new BloomFilterDuplicateRemover(100000)))
                .thread(1)
                .addPipeline(dataPipeline)
                .run();
    }

//    public static void main(String[] args) {
//        new NewsProcesser().runCrawler();
//    }
}
